# -*- coding: utf-8 -*-
import scrapy

import re
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class ChangchunSpider(scrapy.Spider):
    name = 'changchun'
    allowed_domains = ['ccfdw.net']
    custom_settings = {
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 400, 404, 408, 407, 403],
    }
    start_urls = [
        'http://www.ccfdw.net/ecdomain/ccfcjwz/wsba/ysxk/search.jsp?xkzh=20&xkz_num=&xmmc=&sfdw=&flag=1&nowPage=1']
    permit_detail_url_temp = 'http://www.ccfdw.net/ecdomain/ccfcjwz/wsba/ysxk/searchByInfoId.jsp?id={}'  # GET 预售许可证详情url
    project_detail_url_temp = 'http://www.ccfdw.net/ecdomain/lpcs/xmxx/xmjbxxinfo.jsp?Id_xmxq={}'  # GET 项目详情url
    building_li_url_temp = 'http://www.ccfdw.net/ecdomain/lpcs/xmxx/loulist.jsp?Id_xmxq={}'  # GET 楼栋列表url
    building_detail_url_temp = 'http://www.ccfdw.net/ecdomain/lpcs/xmxx/lpbxx_new.jsp?lid={}'  # GET 楼栋详情url
    room_detail_url_temp = 'http://www.ccfdw.net/ecdomain/lpcs/xmxx/huxx.jsp?hid={}&lid={}'  # GET 房号详情url

    def parse(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        page_num = 1 if not response.meta.get('page_num', None) else response.meta.get('page_num', None)
        tr_list = response.xpath("//b[contains(text(),'许可证')]/../../following-sibling::tr[position()!=last()]")
        if len(tr_list):
            for tr_obj in tr_list:
                item_eg = FdcEstateGuidItem()
                item_eg['preSalePermit'] = tr_obj.xpath("./td[1]/a/text()").extract_first()  # 预售许可证
                item_eg['permitId'] = self.get_permit_id(tr_obj.xpath("./td[1]/a/@onclick").extract_first())  # 许可证id
                item_eg['projectName'] = tr_obj.xpath("./td[2]/text()").extract_first()  # 项目名称
                item_eg['developerName'] = tr_obj.xpath("./td[3]/text()").extract_first()  # 开发商

                # 构造请求，获取预售许可证详情
                if item_eg['permitId']:
                    yield scrapy.Request(
                        self.permit_detail_url_temp.format(item_eg['permitId']),
                        callback=self.parse_permit_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=10,
                        dont_filter=False,
                    )
                else:
                    logger.error('第{}页-{}-{} 预售许可证id提取为空，无法获取详情页面'.format(page_num, item_eg['projectName'],
                                                                          item_eg['preSalePermit']))

        # 翻页
        next_page_url = response.xpath("//a[contains(text(),'下一页')]/@href").extract_first()
        next_page_num = page_num + 1
        if next_page_url:
            yield response.follow(
                next_page_url,
                meta=dict(page_num=deepcopy(next_page_num), ),
                dont_filter=True,
            )

    def parse_permit_detail(self, response):
        """
        获取预售许可证详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['preSalePermit'] = response.xpath(
            "//td[contains(text(),'预售许可证编号')]/following-sibling::td[1]/text()").extract_first()  # 预售许可证
        item_eg['certDate'] = response.xpath(
            "//td[contains(text(),'发证日期')]/following-sibling::td[1]/text()").extract_first()  # 发证日期
        item_eg['projectName'] = response.xpath(
            "//td[contains(text(),'项目名称')]/following-sibling::td/a/text()").extract_first()  # 项目名称
        item_eg['projectId'] = self.get_project_id(
            response.xpath("//td[contains(text(),'项目名称')]/following-sibling::td/a/@onclick").extract_first())  # 项目id
        item_eg['projectAddress'] = response.xpath(
            "//td[contains(text(),'项目地址')]/following-sibling::td[1]/text()").extract_first()  # 项目地址
        item_eg['developerName'] = response.xpath(
            "//td[contains(text(),'开发')]/following-sibling::td[1]/text()").extract_first()  # 开发商
        item_eg['preSaleBuilding'] = response.xpath(
            "//td[contains(text(),'预售范围')]/following-sibling::td[1]/text()").extract_first()  # 预售楼栋
        item_eg['preSaleArea'] = response.xpath(
            "//td[contains(text(),'预售总建筑面积')]/following-sibling::td[1]/text()").extract_first()  # 预售面积
        item_eg['permitResidenceArea'] = response.xpath(
            "//td[contains(text(),'其中住宅建筑面积')]/following-sibling::td[1]/text()").extract_first()  # 许可证住宅面积
        item_eg['residenceNum'] = response.xpath(
            "//td[contains(text(),'住宅套数')]/following-sibling::td/text()").extract_first()  # 住宅套数

        # 构造请求，获取项目详情信息
        if item_eg['projectId']:
            yield scrapy.Request(
                self.project_detail_url_temp.format(item_eg['projectId']),
                meta=dict(item_eg=deepcopy(item_eg), ),
                callback=self.parse_project_detail,
                dont_filter=True,  # 关闭过滤
                priority=20,
            )
        else:
            logger.error('{}-{} 项目id提取为空，无法获取项目详情'.format(item_eg['projectName'], item_eg['preSalePermit']))

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['openingDate'] = response.xpath(
            "//td[contains(text(),'开盘日期')]/following-sibling::td[1]/text()").extract_first()  # 开盘日期
        item_eg['greeningRate'] = response.xpath(
            "//td[contains(text(),'绿化率')]/following-sibling::td[1]/text()").extract_first()  # 绿化率
        yield item_eg

        # 构造请求，获取楼栋列表
        yield scrapy.Request(
            self.building_li_url_temp.format(item_eg['projectId']),
            meta=dict(item_eg=deepcopy(item_eg), ),
            callback=self.parse_bulding_li,
            dont_filter=False,  # 开启过滤
            priority=30,
        )

    def parse_bulding_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        building_obj_li = response.xpath("//div[@id='content']/div/a")
        if building_obj_li:
            for building_obj in building_obj_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']
                item_bd['projectId'] = item_eg['projectId']
                item_bd['blockName'] = building_obj.xpath(".//strong/text()").extract_first()  # 楼栋名字
                item_bd['buildingId'] = self.get_building_id(building_obj.xpath("./@onclick").extract_first())  # 楼栋id
                if item_bd['buildingId']:
                    # 构造请求，获取楼栋详情
                    yield scrapy.Request(
                        self.building_detail_url_temp.format(item_bd['buildingId']),
                        meta=dict(item_bd=deepcopy(item_bd), ),
                        callback=self.parse_building_detail,
                        dont_filter=False,  # 开启过滤
                        priority=40,
                    )
                else:
                    logger.error(
                        '{}/{}-{} 楼栋id提取为空'.format(item_bd['projectName'], item_bd['projectId'], item_bd['blockName']))
            del item_eg
        else:
            logger.error(
                '{}/{} 楼栋列表获取为空'.format(item_eg['projectName'], item_eg['projectId']))

    def parse_building_detail(self, response):
        """
        获取楼栋详情
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_bd['preSalePermit'] = response.xpath(
            "//td[contains(text(),'预售许可证')]/following-sibling::td[1]//a/text()").extract_first()  # 预售许可证
        item_bd['buildingStructure'] = response.xpath(
            "//td[contains(text(),'结构')]/following-sibling::td[1]//text()").extract_first()  # 建筑结构
        item_bd['isCompletionAcceptance'] = response.xpath(
            "//td[contains(text(),'现房')]/following-sibling::td[1]//text()").extract_first()  # 现房/期房
        item_bd['floorOverGroundNo'] = response.xpath(
            "//td[contains(text(),'地上层数')]/following-sibling::td[1]//text()").extract_first()  # 地上层数
        item_bd['floorUnderGroundNo'] = response.xpath(
            "//td[contains(text(),'地下层数')]/following-sibling::td[1]//text()").extract_first()  # 地下层数
        item_bd['buildingTotalArea'] = response.xpath(
            "//td[contains(text(),'总建筑面积')]/following-sibling::td[1]//text()").extract_first()  # 总建筑面积
        item_bd['totalRoomNo'] = response.xpath(
            "//td[contains(text(),'总套数')]/following-sibling::td[1]//text()").extract_first()  # 总套数
        item_bd['buildingNo'] = response.xpath(
            "//td[contains(text(),'楼栋栋号')]/following-sibling::td[1]//text()").extract_first()  # 楼栋号
        item_bd['roomUse'] = response.xpath(
            "//td[contains(text(),'房屋用途')]/following-sibling::td[1]//text()").extract_first()  # 房屋用途
        yield item_bd

        # 提取房号列表
        room_obj_li = response.xpath("//table[@class='T_song12bk2']//td[@bgcolor]")
        if room_obj_li:
            for room_obj in room_obj_li:
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']
                item_rm['projectId'] = item_bd['projectId']
                item_rm['preSalePermit'] = item_bd['preSalePermit']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['roomNo'] = room_obj.xpath(".//a/text()").extract_first()  # 房号
                item_rm['roomId'] = self.get_room_id(room_obj.xpath(".//a/@onclick").extract_first())  # 房屋id
                item_rm['saleStatus'] = self.get_room_status(room_obj.xpath("./@bgcolor").extract_first())  # 销售状态
                # 构造请求，获取房号详情
                if item_rm['roomId']:
                    yield scrapy.Request(
                        self.room_detail_url_temp.format(item_rm['roomId'], item_rm['buildingId']),
                        callback=self.parse_room_detail,
                        meta=dict(item_rm=deepcopy(item_rm), ),
                        dont_filter=False,  # 开启过滤
                        priority=50,
                    )
                else:
                    logger.error(
                        '{}-{}-{}-{} 房号id提取为空'.format(item_rm['projectName'], item_rm['preSalePermit'],
                                                      item_rm['blockName'], item_rm['roomNo']))
        else:
            logger.error(
                '{}-{}-{}/{} 房号列表提取为空'.format(item_bd['projectName'], item_bd['preSalePermit'], item_bd['blockName'],
                                              item_bd['buildingId']))
        del item_bd

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['unitNo'] = response.xpath(
            "//td[contains(text(),'单元号')]/following-sibling::td[1]/text()").extract_first()  # 单元号
        item_rm['roomFloor'] = response.xpath(
            "//td[contains(text(),'所在楼层')]/following-sibling::td[1]/text()").extract_first()  # 房屋楼层
        item_rm['buildingStructure'] = response.xpath(
            "//td[contains(text(),'工程结构')]/following-sibling::td[1]/text()").extract_first()  # 建筑结构
        item_rm['roomUse'] = response.xpath(
            "//td[contains(text(),'房屋用途')]/following-sibling::td[1]/text()").extract_first()  # 房屋用途
        item_rm['planningUse'] = response.xpath(
            "//td[contains(text(),'规划用途')]/following-sibling::td[1]/text()").extract_first()  # 规划用途
        item_rm['forecastBuildArea'] = response.xpath(
            "//td[contains(text(),'预测建筑面积')]/following-sibling::td[1]/text()").extract_first()  # 预测建筑面积
        item_rm['forecastInnerArea'] = response.xpath(
            "//td[contains(text(),'预测套内')]/following-sibling::td[1]/text()").extract_first()  # 预测套内面积
        item_rm['forecastSharedArea'] = response.xpath(
            "//td[contains(text(),'预测分摊')]/following-sibling::td[1]/text()").extract_first()  # 预测分摊面积
        item_rm['forecastVerandaArea'] = response.xpath(
            "//td[contains(text(),'预测阳台')]/following-sibling::td[1]/text()").extract_first()  # 预测阳台面积
        item_rm['isMortgage'] = response.xpath(
            "//td[contains(text(),'是否抵押')]/following-sibling::td[1]/text()").extract_first()  # 是否抵押
        item_rm['isSealedOff'] = response.xpath(
            "//td[contains(text(),'是否查封')]/following-sibling::td[1]/text()").extract_first()  # 是否查封
        item_rm['recordUnitPrice'] = response.xpath(
            "//td[contains(text(),'预售价格')]/following-sibling::td[1]/text()").extract_first()  # 备案单价
        yield item_rm

    @staticmethod
    def get_permit_id(permit: str):
        """
        提取预售许可证id
        :param permit:
        :return:
        """
        try:
            permit = permit.replace('\"', '\'')
            permit_id = re.findall(r"'(.*?)'", permit)[0]
            assert permit_id
        except Exception as e:
            logger.error(f'{permit},预售许可证id提取出错,error:{e}')
            return
        else:
            return permit_id

    @staticmethod
    def get_project_id(project_str: str):
        """
        提取项目id
        :param project_str:
        :return:
        """
        try:
            project_str = project_str.replace('\"', '\'')
            project_id = re.findall(r"'(.*?)'", project_str)[0]
            assert project_id
        except Exception as e:
            logger.error(f'{project_str},项目id提取出错,error:{e}')
            return
        else:
            return project_id

    @staticmethod
    def get_building_id(building_str: str):
        """
        提取楼栋id
        :param building_str:
        :return:
        """
        try:
            building_id = re.findall(r"'(.*?)'", building_str)[0]
            assert building_id
        except Exception as e:
            logger.error(f'{building_str},楼栋id提取出错,error:{e}')
            return
        else:
            return building_id

    @staticmethod
    def get_room_id(room_str: str):
        """
        提取房号id
        :param room_str:
        :return:
        """
        try:
            room_id = re.findall(r"'(.*?)'", room_str)[0]
            assert room_id
        except Exception as e:
            logger.error(f'{room_str},房号id提取出错,error:{e}')
            return
        else:
            return room_id

    @staticmethod
    def get_room_status(bgcolor: str):
        """
        提取房号销售状态
        :param bgcolor:
        :return:
        """
        status_dict = {
            '#00c112': '可售',
            '#04fffa': '回迁房',
            '#4a036e': '已预告抵押',
            '#9e42cb': '已预告',
            '#CCFFFF': '正在受理',
            '#bbbea9': '自留',
            '#d37887': '已查封、已冻结、已限制',
            '#d4090d': '已备案',
            '#f0ef79': '已售',
            '#fbfdff': '不可售',
            '#ffc8ff': '已办产权',
            '#ffedf1': '已抵押',
        }
        try:
            room_status = status_dict[bgcolor]
        except:
            return
        else:
            return room_status